# 该网址中报告了2021年QS世界大学排名1200强榜单： https://www.dxsbb.com/news/16131.html 。
# 请对该网址的榜单内容进行抓取和分析。要求： 1. 榜单中的排名有些地方比较笼统，请根据出现顺序进行强制
# 给定排序； 2. 统计出各国家的大学上榜次数； 3. 如果将排名作为参考，同时考虑上榜的大学数量，将各国家的
# 大学进行教育质量统计的话，统计出各国家的教育质量数据，并给出原则。 4. 请将以上结果以表格的形式输出。
# 有余力的同学可以参考第十章内容，可尝试对该类问题的处理问题制作一个GUI操作界面。
import re
import requests
import logging
import csv
import pandas as pd
import numpy as np

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
url = "https://www.dxsbb.com/news/16131.html"
user_agent = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML"
              ", like Gecko) Chrome/118.0.0.0 Safari/537.36 Edg/118.0.2088.76")
header = {
    "User-Agent": user_agent
}
# 爬取网页
receive_html = requests.get(url=url, headers=header)
receive_html.encoding = receive_html.apparent_encoding
receive_text = receive_html.text
unit_re = '.*?<div class="tablebox">(.*?)</div>'
# 正则匹配
web_list = re.match(unit_re, receive_text, re.S).group(1)
# 正则提取
extract_info = re.findall('<td.*?>(.*?)</td>', web_list, re.S)
# 原始数据csv
file = open('crudedata.csv', encoding='utf8', newline='', mode='w')
name = [extract_info[0], extract_info[1], extract_info[2]]
# logger.info(name)
# 删除表头
del extract_info[0:3]
# logger.info(extract_info)
w = csv.DictWriter(file, name)
w.writeheader()

# 处理 写入csv
while len(extract_info) != 0:
    slice_data = extract_info[0:3]
    if '<' in slice_data[1]:
        formula_re = '(.*?)<.*>(.*?)<.*>(.*)'
        slice1 = re.findall(formula_re, slice_data[1], re.S)[0]
        # logger.info(slice1)
        st = ''
        for s in slice1:
            st = st + s
        # logger.info(st)
        slice_data[1] = st
    # logger.info(slice_data)
    # 将列表对象和列名列表组合成字典对象
    my_dict = dict(zip(name, slice_data))
    w.writerow(my_dict)
    del extract_info[0:3]

file.close()

# 处理表格数据
pd_data = pd.read_csv('crudedata.csv')
# 删除指定列 重新排序
pd_data.drop(columns=["排名"])
pd_data["排名"] = range(1, len(pd_data) + 1)
pd_data = pd_data[['排名', '学校名称', '国家/地区']]
pddatas = pd_data['国家/地区'].value_counts()
pdd = pddatas.rename_axis("nation").reset_index(name="counts")
pd_data['nation'] = pdd['nation']
pd_data['counts'] = pdd['counts']


def give_standard_of_edu(college_number):
    edu_std = ''
    if college_number >= 1 or college_number <= 13:
        edu_std = '良好'
    elif college_number > 13:
        edu_std = '优秀'
    else:
        edu_std = '差'
    return edu_std


pd_data["level"] = pd_data['counts'].map(give_standard_of_edu)
pd_data.to_csv("processdata.csv", mode='w', index=False, header=True)



